package com.ustadzmenjawab.crawler;

import java.io.IOException;
import java.util.HashSet;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.ustadzmenjawab.shared.Article;

public class UstadzMenjawabRepublika extends CrawlerRoot {
	public static final String HOME_URL = "http://www.republika.co.id/kanal/konsultasi/ustadz-menjawab";
	public static final String PATTERN = "republika.co.id";
	public static void main(String [] args){
		try {
			int maxListPage = 5;
			UstadzMenjawabRepublika cr = new UstadzMenjawabRepublika();
			HashSet<String> visited = new HashSet<String>();
			for(int i = 0; i < maxListPage;i++){
				Document doc = Jsoup.connect(HOME_URL+"/"+(i*10)).timeout(0).get();
				Elements articlesLs = doc.select("a[href*="+PATTERN+"]");
				for(Element link:  articlesLs){
					try{
						String absHref = link.attr("abs:href");
						if(!visited.contains(absHref)){
							visited.add(absHref);
							if(cr.isRelevantURL(absHref)){
								Document page = Jsoup.connect(absHref).timeout(0).get();
								String content = page.html();
								if(cr.isRelevantPage(content)){
									System.out.println(absHref +" is a relevant page");
									if(!cr.isUrlExist(absHref)){
										cr.insertContent(absHref, content);
									}
								}else{
									System.out.println(absHref +" is not a relevant page");
								}
							}
						}
					}catch(Exception e){
						System.out.println(" exception "+e.getClass() +" :" + e.getMessage());
					}
				}
			}
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
	
	@Override
	public boolean isRelevantURL(String url) {
		if(url.contains("/ustadz-menjawab/"))
			return true;
		return false;
	}

	@Override
	public boolean isRelevantPage(String content) {
		if(content.contains("<div id=\"newstext\">")){
			return true;
		}
		return false;
	}

	@Override
	public String getTitle(String body) {
		Document doc = Jsoup.parse(body);
		Element p = doc.select("h1").first();
		return p.text();
	}

	@Override
	public String getQuestion(String body) {
//		int istart = body.indexOf("<div id=\"newstext\">");
		body = getArticle(body);
		int iend = body.indexOf("Terima kasih");
		if(iend == -1){
			iend = body.indexOf("terima kasih");
		}if(iend == -1){
			iend = body.indexOf("makasih");
		}if(iend == -1){
			iend = body.indexOf("Makasih");
		}if(iend == -1){
			iend = body.indexOf("Jawaban");
		}if(iend == -1){
			iend = body.indexOf("jawaban");
		}if(iend == -1){
			iend = body.indexOf("Jawab");
		}if(iend == -1){
			iend = body.indexOf("jawab");
		}if(iend == -1){
			iend = body.indexOf("Wassalam");
		}if(iend == -1){
			iend = body.indexOf("wassalam");
		}
		if(iend > -1){
			return body.substring(0,iend);
		}else{
			return "";
		}
	}

	@Override
	public String getArticle(String body) {
		Document doc = Jsoup.parse(body);
		Element p = doc.select("div[id*=newstext]").first();
		return p.text();
	}

	@Override
	public boolean insertContent(String url, String content) {
		try{
			Article article = new Article();
			String title = getTitle(content);
			if(isTitleExist(title, "republika.co.id")){
				return false;
			}
			article.setJudul(title);
			article.setKategori("shariah");
			article.setSumber("republika.co.id");
			article.setTahun("2011");
			article.setUrl(url);
			article.setUstadz("Konsultasi Syariah Republika");
			article.setArtikel(getArticle(content));
			article.setPertanyaan(getQuestion(content));
			db.insertArticle(article);
			return true;
		}catch(Exception e){
			System.out.println(" exception "+e.getClass() +" :" + e.getMessage());
			e.printStackTrace();
			insertFailure(url,"exception "+e.getClass() +" :" + e.getMessage());
			return false;
		}
	}

}
